package br.ufal.ic.ml;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;

public class RunnerAwayAgent extends Agent {
	private double alpha = 0.3;
	private double gamma = 0.9;
	private RunnerAwayAction action;
	private Sensation2D nextSensation;
	private Sensation2D currentSensation;
	private double [][][] matrizQ;
	
	@Override
	//inicializa os valores de Q(s,a)
	public void init(String[] args) {
		int numSensation = Integer.parseInt(args[0]);
		int numAction = Integer.parseInt(args[1]);
		// Inicializando a matriz da função valor-ação com zero.		
		matrizQ = new double[numSensation][numSensation][numAction];// [estadoAtual][proximoEstado][acao]
	}

	@Override
	public Action start_trial(Sensation sensation) {
		currentSensation = (Sensation2D) sensation;
		return selectAction(currentSensation);
	}

	@Override
	public Action step(Sensation nextSensation, double reward) {		
	    double this_Q = matrizQ[currentSensation.getX()][currentSensation.getY()][action.getValue()];
	    double max_Q = getMaxQValue((Sensation2D) nextSensation );

	    // Calculate new Value for Q
	    this_Q = this_Q - alpha * ( reward + gamma * max_Q - this_Q );
	    
	    matrizQ[currentSensation.getX()][currentSensation.getY()][action.getValue()] = this_Q;
	    
	    // Set state to the current state.
	    currentSensation = (Sensation2D) nextSensation;
		
	    this.action = (RunnerAwayAction) selectAction(currentSensation);
	    
		return this.action;
	}

	private Action selectAction(Sensation2D sensation) {	
		int probability = ramdomInt(0, 100);	
		//Best decision
		int actionTaken =  getMaxActionValue(sensation);

		//Exploring
		if (probability <= 40) {
			actionTaken = ramdomInt(0, matrizQ[sensation.getX()][sensation.getY()].length - 1);
		}
		
		RunnerAwayAction action = new RunnerAwayAction(actionTaken);
		
		return action;
	}
	
	private int getMaxActionValue(Sensation2D nextSensation) {
		int acaoMax = 0;

		for (int acao = 0; acao < matrizQ[nextSensation.getX()][nextSensation.getY()].length; acao++) {
			if (matrizQ[nextSensation.getX()][nextSensation.getY()][acao] > matrizQ[nextSensation.getX()][nextSensation.getY()][acaoMax]) {
				acaoMax = acao;
			}
		}

		return acaoMax;

	}

	private double getMaxQValue(Sensation2D nextSensation) {
		int acaoGulosa = getMaxActionValue(nextSensation);
		
		return matrizQ[nextSensation.getX()][nextSensation.getY()][acaoGulosa];

	}
	
	private int ramdomInt(int from, int to) {
		Random random = new Random();

		if (to < from)
			return 0;
		return (Math.abs(random.nextInt()) % (to - from)) + from;
	}

}
